/*


Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"encoding/base64"
	"fmt"
	ckev1alpha1 "iop.inspur.com/api/v1alpha1"
	"iop.inspur.com/constants"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"net"
	"os"
	"strconv"
	"time"
)

func newComponentTerminateJob(port int32, component *ckev1alpha1.CkeComponent, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(3), int32(1), int32(0600)
	CkeadmImageName := GetCkeadmImageName(cluster)
	ScriptImageName := GetScriptImageNameFromSpec(cluster, component)
	components := []*ckev1alpha1.CkeComponent{component}
	waitAddress := GetCkeadmWaitHealthyNode(nodes, port)
	if waitAddress == constants.EmptyString { // 当找不到 health 的 ckeadmwait 时，采用老的方式获取 address
		waitAddress = nodes[0].Spec.Address
	}

	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s", ComponentTerminateJobAppLabel, component.Name),
			Namespace: component.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(component, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeComponent"}),
			},
			Labels: map[string]string{
				"app":           ComponentTerminateJobAppLabel,
				"componentName": component.Name,
				"clusterID":     cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s-%s", ComponentTerminateJobAppLabel, component.Name, RandomUID()),
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"/ckeadm",
								"component",
								"delete",
								"--cluster",
								Base64ContextForCluster(cluster),
								"--node",
								Base64ContextForNodes(nodes),
								"--component",
								Base64ContextForComponent(components),
								"--address",
								waitAddress,
								"--port",
								strconv.FormatInt(int64(port), 10),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/root/.ssh/",
									Name:      "cke-certs",
								},
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					InitContainers: []corev1.Container{
						{
							Name:            ComponentContainerName,
							Image:           ScriptImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", ScriptPathInScriptImage, "/script/"+component.Name),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "share-volume",
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{},
							},
						},
					},
				},
			},
		},
	}
}

func newComponentInstallJob(port int32, component *ckev1alpha1.CkeComponent, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(3), int32(1), int32(0600)
	CkeadmImageName := GetCkeadmImageName(cluster)
	ScriptImageName := GetScriptImageNameFromSpec(cluster, component)
	components := []*ckev1alpha1.CkeComponent{component}
	waitAddress := GetCkeadmWaitHealthyNode(nodes, port)
	if waitAddress == constants.EmptyString { // 当找不到 health 的 ckeadmwait 时，采用老的方式获取 address
		waitAddress = nodes[0].Spec.Address
	}

	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s", ComponentInstallJobAppLabel, component.Name),
			Namespace: component.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(component, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeComponent"}),
			},
			Labels: map[string]string{
				"app":           ComponentInstallJobAppLabel,
				"componentName": component.Name,
				"clusterID":     cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s-%s", ComponentInstallJobAppLabel, component.Name, RandomUID()),
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"/ckeadm",
								"component",
								"create",
								"--cluster",
								Base64ContextForCluster(cluster),
								"--node",
								Base64ContextForNodes(nodes),
								"--component",
								Base64ContextForComponent(components),
								"--address",
								waitAddress,
								"--port",
								strconv.FormatInt(int64(port), 10),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/root/.ssh/",
									Name:      "cke-certs",
								},
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					InitContainers: []corev1.Container{
						{
							Name:            ComponentContainerName,
							Image:           ScriptImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", ScriptPathInScriptImage, "/script/"+component.Name),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "share-volume",
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{},
							},
						},
					},
				},
			},
		},
	}
}

func newComponentUpgradeJob(port int32, component *ckev1alpha1.CkeComponent, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(3), int32(1), int32(0600)
	CkeadmImageName := GetCkeadmImageName(cluster)
	ScriptImageName := GetScriptImageNameFromSpec(cluster, component)
	components := []*ckev1alpha1.CkeComponent{component}
	waitAddress := GetCkeadmWaitHealthyNode(nodes, port)
	if waitAddress == constants.EmptyString { // 当找不到 health 的 ckeadmwait 时，采用老的方式获取 address
		waitAddress = nodes[0].Spec.Address
	}
	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s", ComponentUpgradeJobAppLabel, component.Name),
			Namespace: component.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(component, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeComponent"}),
			},
			Labels: map[string]string{
				"app":           ComponentUpgradeJobAppLabel,
				"componentName": component.Name,
				"clusterID":     cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s-%s", ComponentUpgradeJobAppLabel, component.Name, RandomUID()),
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"/ckeadm",
								"component",
								"upgrade",
								"--cluster",
								Base64ContextForCluster(cluster),
								"--node",
								Base64ContextForNodes(nodes),
								"--component",
								Base64ContextForComponent(components),
								"--address",
								waitAddress,
								"--port",
								strconv.FormatInt(int64(port), 10),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/root/.ssh/",
									Name:      "cke-certs",
								},
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					InitContainers: []corev1.Container{
						{
							Name:            ComponentContainerName,
							Image:           ScriptImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", ScriptPathInScriptImage, "/script/"+component.Name),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "share-volume",
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{},
							},
						},
					},
				},
			},
		},
	}
}

func newComponentUpdateJob(port int32, component *ckev1alpha1.CkeComponent, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(3), int32(1), int32(0600)
	CkeadmImageName := GetCkeadmImageName(cluster)
	ScriptImageName := GetScriptImageNameFromSpec(cluster, component)
	components := []*ckev1alpha1.CkeComponent{component}
	waitAddress := GetCkeadmWaitHealthyNode(nodes, port)
	if waitAddress == constants.EmptyString { // 当找不到 health 的 ckeadmwait 时，采用老的方式获取 address
		waitAddress = nodes[0].Spec.Address
	}
	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s", ComponentUpdateJobAppLabel, component.Name),
			Namespace: component.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(component, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeComponent"}),
			},
			Labels: map[string]string{
				"app":           ComponentUpdateJobAppLabel,
				"componentName": component.Name,
				"clusterID":     cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s-%s", ComponentUpdateJobAppLabel, component.Name, RandomUID()),
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"/ckeadm",
								"component",
								"update",
								"--cluster",
								Base64ContextForCluster(cluster),
								"--node",
								Base64ContextForNodes(nodes),
								"--component",
								Base64ContextForComponent(components),
								"--address",
								waitAddress,
								"--port",
								strconv.FormatInt(int64(port), 10),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/root/.ssh/",
									Name:      "cke-certs",
								},
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					InitContainers: []corev1.Container{
						{
							Name:            ComponentContainerName,
							Image:           ScriptImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", ScriptPathInScriptImage, "/script/"+component.Name),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "share-volume",
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{},
							},
						},
					},
				},
			},
		},
	}
}

//	新节点初始化使用的ansible-job
func newComponentScaleUpAnsibleJob(components []ckev1alpha1.CkeComponent, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(2), int32(1), int32(0600)
	var ImageName = ImageGetFromConfig(&cluster.Spec)
	var CkeConfigContent = Base64ContextForAll(cluster, nodes, components)
	var node = GetNodeByRole(&nodes, constants.ScaleRole)
	var fork = strconv.Itoa(constants.CkeNodeForks)
	var execYAML = constants.ComponentScaleYamlFile

	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s", "scale-nodes-job", RandomUID()),
			Namespace: cluster.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(cluster, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeCluster"}),
			},
			Labels: map[string]string{
				"app":       ScaleJobAppLabel,
				"nodeName":  node.Name,
				"clusterID": cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: "scale-nodes-pods",
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{{
						Name:            "cke-scale-nodes",
						Image:           ImageName,
						ImagePullPolicy: corev1.PullIfNotPresent,
						Command: []string{
							"/bin/playbook",
							"--execyml",
							execYAML,
							"--config",
							CkeConfigContent,
							"--fork",
							fork,
							"--logpath",
							constants.LogsInpodDir,
							"--logname",
							fmt.Sprintf("scale-%s", node.Name),
						},
						Env: []corev1.EnvVar{
							{
								Name:  "ANSIBLE_HOST_KEY_CHECKING",
								Value: "Flase",
							},
							{
								Name:  "ANSIBLE_GATHERING",
								Value: "explicit",
							},
						},
						VolumeMounts: []corev1.VolumeMount{
							{
								MountPath: "/root/.ssh/",
								Name:      "cke-certs",
							},
							{
								MountPath: constants.LogsInpodDir,
								Name:      "ckelogs",
							},
						},
					}},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "ckelogs",
							VolumeSource: corev1.VolumeSource{
								HostPath: &corev1.HostPathVolumeSource{
									Path: constants.ScaleLogDir,
									Type: &constants.MountPathDirType,
								},
							},
						},
					},
				},
			},
		},
	}
}

func newComponentScaleUpJob(port int32, component *ckev1alpha1.CkeComponent, addNode string, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(3), int32(1), int32(0600)
	CkeadmImageName := GetCkeadmImageName(cluster)
	ScriptImageName := GetScriptImageNameFromStatus(cluster, component)
	components := []*ckev1alpha1.CkeComponent{component}
	for k, node := range nodes {
		if node.Name == addNode {
			nodes[k].Spec.Roles = append(node.Spec.Roles, CkeNodeScaleupRole)
		}
	}
	waitAddress := GetCkeadmWaitHealthyNode(nodes, port)
	if waitAddress == constants.EmptyString { // 当找不到 health 的 ckeadmwait 时，采用老的方式获取 address
		if nodes[0].Name != addNode {
			waitAddress = nodes[0].Spec.Address
		} else {
			waitAddress = nodes[1].Spec.Address
		}
	}
	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s-%s", ComponentScaleupJobAppLabel, component.Name, addNode),
			Namespace: component.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(component, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeComponent"}),
			},
			Labels: map[string]string{
				"app":           ComponentScaleupJobAppLabel,
				"componentName": component.Name,
				"addNodeName":   addNode,
				"clusterID":     cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s-%s-%s", ComponentScaleupJobAppLabel, component.Name, addNode, RandomUID()),
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"/ckeadm",
								"component",
								"scaleup",
								"--cluster",
								Base64ContextForCluster(cluster),
								"--node",
								Base64ContextForNodes(nodes),
								"--component",
								Base64ContextForComponent(components),
								"--nodeName",
								addNode,
								"--address",
								waitAddress,
								"--port",
								strconv.FormatInt(int64(port), 10),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/root/.ssh/",
									Name:      "cke-certs",
								},
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					InitContainers: []corev1.Container{
						{
							Name:            ComponentContainerName,
							Image:           ScriptImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", ScriptPathInScriptImage, "/script/"+component.Name),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "share-volume",
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{},
							},
						},
					},
				},
			},
		},
	}
}

func newComponentScaleDownJob(port int32, component *ckev1alpha1.CkeComponent, deleteNode string, nodes []ckev1alpha1.CkeNode, cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com, Model = int32(1), int32(3), int32(1), int32(0600)
	CkeadmImageName := GetCkeadmImageName(cluster)
	ScriptImageName := GetScriptImageNameFromStatus(cluster, component)
	components := []*ckev1alpha1.CkeComponent{component}
	for k, node := range nodes {
		if node.Name == deleteNode {
			nodes[k].Spec.Roles = append(node.Spec.Roles, CkeNodeScaleDownRole)
		}
	}
	waitAddress := GetCkeadmWaitHealthyNode(nodes, port)
	if waitAddress == constants.EmptyString { // 当找不到 health 的 ckeadmwait 时，采用老的方式获取 address
		if nodes[0].Name != deleteNode {
			waitAddress = nodes[0].Spec.Address
		} else {
			waitAddress = nodes[1].Spec.Address
		}
	}
	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-%s-%s", ComponentScaledownJobAppLabel, component.Name, deleteNode),
			Namespace: component.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(component, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeComponent"}),
			},
			Labels: map[string]string{
				"app":            ComponentScaledownJobAppLabel,
				"componentName":  component.Name,
				"deleteNodeName": deleteNode,
				"clusterID":      cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s-%s-%s", ComponentScaledownJobAppLabel, component.Name, deleteNode, RandomUID()),
				},
				Spec: corev1.PodSpec{
					HostNetwork: true,
					NodeSelector: map[string]string{
						"beta.kubernetes.io/os":          "linux",
						"node-role.kubernetes.io/master": "true",
					},
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"/ckeadm",
								"component",
								"scaledown",
								"--cluster",
								Base64ContextForCluster(cluster),
								"--node",
								Base64ContextForNodes(nodes),
								"--component",
								Base64ContextForComponent(components),
								"--nodeName",
								deleteNode,
								"--address",
								waitAddress,
								"--port",
								strconv.FormatInt(int64(port), 10),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/root/.ssh/",
									Name:      "cke-certs",
								},
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					InitContainers: []corev1.Container{
						{
							Name:            ComponentContainerName,
							Image:           ScriptImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", ScriptPathInScriptImage, "/script/"+component.Name),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: "/script/" + component.Name,
									Name:      "share-volume",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "cke-certs",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									DefaultMode: &Model,
									SecretName:  "cke-certs",
								},
							},
						},
						{
							Name: "share-volume",
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{},
							},
						},
					},
				},
			},
		},
	}
}

func newCpCkePatchJob(cluster *ckev1alpha1.CkeCluster) *batchv1.Job {
	var Para, Back, Com = int32(1), int32(3), int32(1)
	CkeadmImageName := GetCkeadmImageName(cluster)
	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      CkePatchCpJobAppLabel,
			Namespace: cluster.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(cluster, schema.GroupVersionKind{Group: ckev1alpha1.GroupVersion.Group, Version: ckev1alpha1.GroupVersion.Version, Kind: "CkeCluster"}),
			},
			Labels: map[string]string{
				"app":       CkePatchCpJobAppLabel,
				"clusterID": cluster.Name,
			},
		},
		Spec: batchv1.JobSpec{
			Parallelism:  &Para,
			BackoffLimit: &Back,
			Completions:  &Com,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("%s-%s", CkePatchCpJobAppLabel, RandomUID()),
				},
				Spec: corev1.PodSpec{
					NodeName: os.Getenv("MY_NODE_NAME"),
					Tolerations: []corev1.Toleration{
						{
							Key:      "CriticalAddonsOnly",
							Operator: "Exists",
						},
						{
							Operator: "Exists",
							Effect:   "NoExecute",
						},
						{
							Operator: "Exists",
							Effect:   "NoSchedule",
						},
					},
					Containers: []corev1.Container{
						{
							Name:            CkeadmContainerName,
							Image:           CkeadmImageName,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command: []string{
								"sh",
								"-c",
								fmt.Sprintf("cp -rf %s %s", CkeadmCkePatchPath+"/*", CkeControllerCkePatchPath),
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									MountPath: CkeControllerCkePatchPath,
									Name:      "ckepatchs",
								},
							},
						},
					},
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Volumes: []corev1.Volume{
						{
							Name: "ckepatchs",
							VolumeSource: corev1.VolumeSource{
								HostPath: &corev1.HostPathVolumeSource{
									Path: CkeControllerCkePatchPath,
									Type: &constants.MountPathDirType,
								},
							},
						},
					},
				},
			},
		},
	}
}

func GetCkeadmWaitHealthyNode(nodes []ckev1alpha1.CkeNode, port int32) string {
	for item := range nodes {
		if DialPortTimeout(nodes[item].Spec.Address, int(port)) {
			return nodes[item].Spec.Address
		}
	}

	return constants.EmptyString
}

func DialPortTimeout(server string, port int) bool {
	if conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", server, port), time.Second); err == nil {
		defer conn.Close()
		return true
	}
	return false
}

func GetCkeadmImageName(cluster *ckev1alpha1.CkeCluster) string {
	return lGetImage(cluster.Spec.Registry.Domain, cluster.Spec.Registry.Port, CkeadmImageRepo,
		cluster.Spec.Version, cluster.Spec.ClusterParameters.ImageManifestEnabled)
}

func GetScriptImageNameFromStatus(cluster *ckev1alpha1.CkeCluster, component *ckev1alpha1.CkeComponent) string {
	return lGetImage(cluster.Spec.Registry.Domain, cluster.Spec.Registry.Port, CkeComponentScriptImageRepo+component.Name,
		component.Status.ImageTag, cluster.Spec.ClusterParameters.ImageManifestEnabled)
}

func GetScriptImageNameFromSpec(cluster *ckev1alpha1.CkeCluster, component *ckev1alpha1.CkeComponent) string {
	return lGetImage(cluster.Spec.Registry.Domain, cluster.Spec.Registry.Port, CkeComponentScriptImageRepo+component.Name,
		component.Spec.ImageTag, cluster.Spec.ClusterParameters.ImageManifestEnabled)
}

func lGetImage(registry string, port int, name, tag string, disableArchTag bool) string {
	var head, body, end = GetRegistry(registry, port), name, tag

	// if enable archtag, add arch information for image. such as xxx-amd64
	if !disableArchTag {
		body = fmt.Sprintf("%s-%s", body, constants.ARCH)
	}

	return fmt.Sprintf("%s/%s:%s", head, body, end)
}

func GetRegistry(registry string, port int) string {
	// if has port, config registry with port as xxxx:xxxx
	if port > 0 {
		return fmt.Sprintf("%s:%d", registry, port)
	}

	return registry
}

func Base64ContextForAll(cluster *ckev1alpha1.CkeCluster, nodes []ckev1alpha1.CkeNode, components []ckev1alpha1.CkeComponent) (base64context string) {
	var out []byte
	var err error
	var content string
	codec, _ := runtime.SerializerInfoForMediaType(serializer.NewCodecFactory(runtime.NewScheme()).SupportedMediaTypes(), "application/yaml")

	if out, err = runtime.Encode(codec.Serializer, cluster); err != nil {
		return ""
	}
	content += fmt.Sprintf("---\n%s\n", string(out))

	for item := range nodes {
		if out, err = runtime.Encode(codec.Serializer, &(nodes)[item]); err != nil {
			return ""
		}
		content += fmt.Sprintf("---\n%s\n", string(out))
	}

	for item := range components {
		if out, err = runtime.Encode(codec.Serializer, &components[item]); err != nil {
			return ""
		}
		content += fmt.Sprintf("---\n%s\n", string(out))
	}

	return base64.StdEncoding.EncodeToString([]byte(content))
}

func Base64ContextForCluster(cluster *ckev1alpha1.CkeCluster) (base64context string) {
	var out []byte
	var err error
	var content string
	codec, _ := runtime.SerializerInfoForMediaType(serializer.NewCodecFactory(runtime.NewScheme()).SupportedMediaTypes(), "application/yaml")

	if out, err = runtime.Encode(codec.Serializer, cluster); err != nil {
		return ""
	}
	content += fmt.Sprintf("---\n%s\n", string(out))

	return base64.StdEncoding.EncodeToString([]byte(content))
}

func Base64ContextForNodes(nodes []ckev1alpha1.CkeNode) (base64context string) {
	var out []byte
	var err error
	var content string
	codec, _ := runtime.SerializerInfoForMediaType(serializer.NewCodecFactory(runtime.NewScheme()).SupportedMediaTypes(), "application/yaml")

	for item := range nodes {
		if out, err = runtime.Encode(codec.Serializer, &(nodes)[item]); err != nil {
			return ""
		}
		content += fmt.Sprintf("---\n%s\n", string(out))
	}

	return base64.StdEncoding.EncodeToString([]byte(content))
}

func Base64ContextForComponent(components []*ckev1alpha1.CkeComponent) (base64context string) {
	var out []byte
	var err error
	var content string
	codec, _ := runtime.SerializerInfoForMediaType(serializer.NewCodecFactory(runtime.NewScheme()).SupportedMediaTypes(), "application/yaml")

	for item := range components {
		if out, err = runtime.Encode(codec.Serializer, components[item]); err != nil {
			return ""
		}
		content += fmt.Sprintf("---\n%s\n", string(out))
	}

	return base64.StdEncoding.EncodeToString([]byte(content))
}
